/*
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov.vf;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

import net.javlov.Action;
import net.javlov.Option;
import net.javlov.Policy;
import net.javlov.State;
import net.javlov.VectorState;
import net.javlov.example.grid.Direction;
import net.javlov.example.grid.GridMove;
import net.javlov.policy.EGreedyPolicy;

public class Test {
	
	//static StateValuePair[] cache;
	static int size = 50000, numOpts = 4, feats = 32;
	static double traces[][], weights[][];
	static int nonzerotraces[][], numNonZero[];
	
	public static void main(String[] args) {

		Map<Integer, Double> traceMap = new HashMap<Integer,Double>(50*feats);
		
		State<int[]> s;
		int[] tile;
		Random rand = new Random();
		int opt;
		long start;
		for ( int r = 0; r < 10; r ++ ) {
			init();
			start = System.currentTimeMillis();
			for ( int i = 0; i < 10000; i++ ) {
				tile = new int[feats];
				for (int j = 0; j < tile.length; j++)
					tile[j] = rand.nextInt(size);
				s = new State<int[]>(tile);
				
				//pick an option
				opt = rand.nextInt(numOpts);
				
				//System.out.println("---------------");
				//System.out.println(numNonZero[opt]);
				//update traces and weights
				//updateTracesNonzero(tile,opt);
				updateTracesNaive(tile,opt);
				//System.out.println(numNonZero[opt]);
	
			}
			System.out.println(Arrays.toString(numNonZero));
			System.out.println(System.currentTimeMillis() - start);
		}
			
		
		/*//actions
		List<Option<? super double[]>> optionPool = new ArrayList<Option<? super double[]>>(4);
		optionPool.add( new GridMove(Direction.north, null) );
		optionPool.add( new GridMove(Direction.east, null) );
		optionPool.add( new GridMove(Direction.south, null) );
		optionPool.add( new GridMove(Direction.west, null) );
				
		QMap vt = new QMap<double[]>(optionPool, 150); //for TD(0)
		TableOptionValues.trackMax = false;
		
		//policy
		Policy<double[]> pi = new EGreedyPolicy<double[]>(vt, 0.1, optionPool);
		
		int[] sizes = new int[]{10,100,1000,10000,100000,500000};
		int size;
		int numtests = 10000000;
		long start;
		VectorState states[];
		VectorState s1, s2;
		OptionValues currValues;
		Option currOpt, lastOpt;
		State lastState;
		double lastVal;
		
		double value;
		for ( int i = 0; i < sizes.length; i++ ) {
			size = sizes[i];
			states = new VectorState[size];
			vt.init();
			start = System.currentTimeMillis();
			
			states[0] = new VectorState(new double[]{0});
			currValues = vt.getValues(states[0]);
			currOpt = pi.getOption(states[0],currValues);
			lastOpt = currOpt;
			lastVal = currValues.get(currOpt);
			lastState = states[0];
			for ( int j = 1; j < size; j++ ) {
				states[j] = new VectorState(new double[]{j});
				currValues = vt.getValues(states[j]);
				currOpt = pi.getOption(states[j],currValues);
				vt.update(lastState, lastOpt, 0.1*(Math.random() + 0.9*currValues.max() - lastVal));
				lastVal = currValues.get(currOpt);
				lastState = states[j];	
			}
			System.out.print(vt.size() + ", " + (System.currentTimeMillis() - start) + ", ");
			
			start = System.currentTimeMillis();
			s1 = states[(int)Math.floor(Math.random() * size)];
			currValues = vt.getValues(s1);
			currOpt = pi.getOption(s1,currValues);
			lastOpt = currOpt;
			lastVal = currValues.get(currOpt);
			lastState = s1;
			for ( int t = 0; t < numtests; t++ ) {
				s2 = states[(int)Math.floor(Math.random() * size)];
				currValues = vt.getValues(s2);
				currOpt = pi.getOption(s2,currValues);
				vt.update(lastState, lastOpt, 0.1*(Math.random() + 0.9*currValues.max() - lastVal));
				s1 = s2;
				lastVal = currValues.get(currOpt);
				lastState = s1;
			}
			System.out.println((System.currentTimeMillis() - start));			
		}*/
	}
	
	static void init() {
		traces = new double[numOpts][size];
		weights = new double[numOpts][size];
		nonzerotraces = new int[numOpts][size];
		numNonZero = new int[numOpts];
	}
	
	static void updateTracesNaive(int[] tile, int opt) {
		//update traces
		for ( int o = 0; o < numOpts; o++ )
			for ( int f = 0; f < size; f++ )
				traces[o][f] *= 0.99; //e = gamma*lambda*e
		
		for ( int f = 0; f < feats; f++ )
			traces[opt][tile[f]] = 1;
		
		//update weights
		for ( int o = 0; o < numOpts; o++ ) {
			for ( int f = 0; f < size; f++ )
				weights[o][f] += Math.random()*traces[o][f];
		}
	}
	
	static void updateTracesNonzero(int[] tile, int opt) {
		//System.out.println("UT " + opt + ":" + numNonZero[opt]);
		//update traces
		for ( int o = 0; o < numOpts; o++ ) {
			for ( int f = numNonZero[o]-1; f >= 0; f-- ) {
				//System.out.print(traces[o][nonzerotraces[o][f]] + "->");
				traces[o][nonzerotraces[o][f]] *= 0.99; //e = gamma*lambda*e
				//System.out.print(traces[o][nonzerotraces[o][f]] + "\n");
				//remove trace if below threshold
				if ( traces[o][nonzerotraces[o][f]] < 0.01 ) {
					traces[o][nonzerotraces[o][f]] = 0;
					nonzerotraces[o][f] = nonzerotraces[o][numNonZero[o]-1]; //copy index from head to current loc
					numNonZero[o]--;
				}
			}
			//System.out.print(o + ":" + numNonZero[o] + ",");
		}
		//System.out.print("\n");
		
		for ( int f = 0; f < feats; f++ ) {
			if ( traces[opt][tile[f]] < 0.01 ) { //only increase nonzero count if trace was 0
				nonzerotraces[opt][numNonZero[opt]++] = tile[f];
			}
			traces[opt][tile[f]] = 1;
		}
		//System.out.println(opt + ":" + numNonZero[opt]);
		
		//update weights
		for ( int o = 0; o < numOpts; o++ ) {
			for ( int f = 0; f < numNonZero[o]; f++ )
				weights[o][nonzerotraces[o][f]] += Math.random()*traces[o][nonzerotraces[o][f]];
		}
		
/*		for ( int i = 0; i < tile.length; i++ )
			weights[opt][tile[i]] += Math.random();*/
	}

}
